From 51b2e3e2151e0fe43a141ca96e7cf54cab26231b Mon Sep 17 00:00:00 2001 From: tsteven4 <13596209+tsteven4@users.noreply.github.com> Date: Fri, 10 Jan 2020 13:12:11 -0700 Subject: [PATCH] fix char string corruption with xcsv. (#455) Any gmsd char string data handled by the xcsv format could be corrupted when main ran cet_convert_strings. The xml format reader/writer use a QTextCodec to handle all char string conversion, but the data was converted again by cet_convert_strings ... garmin_fs_convert to/from ASCII, resulting in corruption. Add a test case for the above. Add xcsv format support for gmsd email field. --- reference/gmsd.unicsv | 3 + reference/gmsd.xcsv | 2 + testo.d/xcsv.test | 22 +++++ xcsv.cc | 12 ++- xcsv_tokens.gperf | 168 +++++++++++++++++++------------------ xcsv_tokens.in | 1 + xmldoc/chapters/styles.xml | 14 +++- 7 files changed, 137 insertions(+), 85 deletions(-) create mode 100644 reference/gmsd.unicsv create mode 100644 reference/gmsd.xcsv diff --git a/reference/gmsd.unicsv b/reference/gmsd.unicsv new file mode 100644 index 000000000..1422bd5e0 --- /dev/null +++ b/reference/gmsd.unicsv @@ -0,0 +1,3 @@ +No,Latitude,Longitude,Name,Description,URL,Address,City,PostalCode,Country,Phone,Email +1,52.523013,13.398808,"MA'LOA Poké Bowl","restaurant","https://www.maloa.com/","Oranienburger Straße 7","Berlin","10178","Deutschland","030-28427238","franchise@maloa.com" +2,55.741225,37.524863,"Verde café","ресторан","http://verde-cafe.ru/","дом, Kutuzovsky Ave, 36, строение 4, подъезд 3","Москва","121170","Россия","+7 499 350-96-44","info@green-company.org" diff --git a/reference/gmsd.xcsv b/reference/gmsd.xcsv new file mode 100644 index 000000000..406c90404 --- /dev/null +++ b/reference/gmsd.xcsv @@ -0,0 +1,2 @@ +52.523013 13.398808 MA'LOA Poké Bowl restaurant https://www.maloa.com/ Oranienburger Straße 7 Berlin Deutschland 030-28427238 10178 franchise@maloa.com +55.741225 37.524863 Verde café ресторан http://verde-cafe.ru/ дом, Kutuzovsky Ave, 36, строение 4, подъезд 3 Москва Россия +7 499 350-96-44 121170 info@green-company.org diff --git a/testo.d/xcsv.test b/testo.d/xcsv.test index 6cb0569a6..7fe416553 100644 --- a/testo.d/xcsv.test +++ b/testo.d/xcsv.test @@ -70,3 +70,25 @@ echo 'IFIELD LON_DECIMAL,"","%f"' >>${TMPDIR}/route1.style echo 'IFIELD ROUTE_NAME,"","%s"' >>${TMPDIR}/route1.style gpsbabel -i xcsv,style=${TMPDIR}/route1.style -f ${REFERENCE}/route/route1.csv -o gpx -F ${TMPDIR}/route1~csv.gpx compare ${REFERENCE}/route/route1~csv.gpx ${TMPDIR}/route1~csv.gpx + +# gmsd fields +echo 'DESCRIPTION gmsd test' >> ${TMPDIR}/gmsd.style +echo 'EXTENSION csv' >> ${TMPDIR}/gmsd.style +echo 'FIELD_DELIMITER TAB' >> ${TMPDIR}/gmsd.style +echo 'RECORD_DELIMITER NEWLINE' >> ${TMPDIR}/gmsd.style +echo 'IFIELD LAT_DECIMAL,"","%f"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD LON_DECIMAL,"","%f"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD SHORTNAME,"","%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD DESCRIPTION,"","%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD URL,"","%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD STREET_ADDR, "", "%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD CITY, "", "%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD COUNTRY, "", "%s"' >> ${TMPDIR}/gmsd.style +#echo 'IFIELD FACILITY, "", "%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD PHONE_NR, "", "%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD POSTAL_CODE, "", "%s"' >> ${TMPDIR}/gmsd.style +echo 'IFIELD EMAIL, "", "%s"' >> ${TMPDIR}/gmsd.style +gpsbabel -i unicsv -f ${REFERENCE}/gmsd.unicsv -o xcsv,style=${TMPDIR}/gmsd.style -F ${TMPDIR}/gmsd.xcsv +compare ${REFERENCE}/gmsd.xcsv ${TMPDIR}/gmsd.xcsv +gpsbabel -i xcsv,style=${TMPDIR}/gmsd.style -f ${REFERENCE}/gmsd.xcsv -o unicsv -F ${TMPDIR}/gmsd.unicsv +compare ${REFERENCE}/gmsd.unicsv ${TMPDIR}/gmsd.unicsv diff --git a/xcsv.cc b/xcsv.cc index e6627c3ac..dadb466b5 100644 --- a/xcsv.cc +++ b/xcsv.cc @@ -93,6 +93,7 @@ enum xcsv_token { XT_CONSTANT, XT_COUNTRY, XT_DESCRIPTION, + XT_EMAIL, XT_EXCEL_TIME, XT_FACILITY, XT_FILENAME, @@ -982,6 +983,11 @@ xcsv_parse_val(const char* s, Waypoint* wpt, const field_map& fmp, GMSD_SET(facility, csv_stringtrim(s, enclosure, 0)); } break; + case XT_EMAIL: { + garmin_fs_t* gmsd = gmsd_init(wpt); + GMSD_SET(email, csv_stringtrim(s, enclosure, 0)); + } + break; case -1: if (strncmp(fmp.key.constData(), "LON_10E", 7) == 0) { wpt->longitude = atof(s) / pow(10.0, atof(fmp.key.constData()+7)); @@ -1705,6 +1711,10 @@ xcsv_waypt_pr(const Waypoint* wpt) garmin_fs_t* gmsd = GMSD_FIND(wpt); buff = QString().sprintf(fmp.printfc.constData(), GMSD_GET(facility, "")); } + case XT_EMAIL: { + garmin_fs_t* gmsd = GMSD_FIND(wpt); + buff = QString().sprintf(fmp.printfc.constData(), GMSD_GET(email, "")); + } break; /* specials */ case XT_FILENAME: @@ -2224,7 +2234,7 @@ ff_vecs_t xcsv_vecs = { xcsv_data_write, nullptr, &xcsv_args, - CET_CHARSET_ASCII, 0, /* CET-REVIEW */ + CET_CHARSET_UTF8, 0, /* conversion to utf8 for gmsd is handled by the stream, don't let csv_convert_strings convert gmsd data */ { nullptr, nullptr, nullptr, xcsv_wr_position_init, xcsv_wr_position, xcsv_wr_position_deinit }, nullptr diff --git a/xcsv_tokens.gperf b/xcsv_tokens.gperf index b2fabcbf7..235530e2c 100644 --- a/xcsv_tokens.gperf +++ b/xcsv_tokens.gperf @@ -32,7 +32,7 @@ #line 1 "xcsv_tokens.in" struct xt_mapping {const char *name; int xt_token; }; -#define TOTAL_KEYWORDS 86 +#define TOTAL_KEYWORDS 87 #define MIN_WORD_LENGTH 3 #define MAX_WORD_LENGTH 21 #define MIN_HASH_VALUE 7 @@ -111,177 +111,179 @@ Perfect_Hash::in_word_set (const char *str, size_t len) { static struct xt_mapping wordlist[] = { -#line 41 "xcsv_tokens.in" +#line 42 "xcsv_tokens.in" {"LAT_DIR", XT_LAT_DIR}, -#line 69 "xcsv_tokens.in" +#line 70 "xcsv_tokens.in" {"POWER", XT_POWER}, -#line 45 "xcsv_tokens.in" +#line 46 "xcsv_tokens.in" {"LAT_DDMMDIR", XT_LAT_DDMMDIR}, -#line 51 "xcsv_tokens.in" +#line 52 "xcsv_tokens.in" {"LON_DIR", XT_LON_DIR}, -#line 38 "xcsv_tokens.in" +#line 39 "xcsv_tokens.in" {"LAT_DECIMALDIR", XT_LAT_DECIMALDIR}, -#line 54 "xcsv_tokens.in" +#line 55 "xcsv_tokens.in" {"LON_DDMMDIR", XT_LON_DDMMDIR}, -#line 48 "xcsv_tokens.in" +#line 49 "xcsv_tokens.in" {"LON_DECIMALDIR", XT_LON_DECIMALDIR}, -#line 58 "xcsv_tokens.in" +#line 59 "xcsv_tokens.in" {"NET_TIME", XT_NET_TIME}, -#line 30 "xcsv_tokens.in" +#line 31 "xcsv_tokens.in" {"HEART_RATE", XT_HEART_RATE}, -#line 73 "xcsv_tokens.in" +#line 74 "xcsv_tokens.in" {"STREET_ADDR", XT_STREET_ADDR}, #line 6 "xcsv_tokens.in" {"CADENCE", XT_CADENCE}, -#line 46 "xcsv_tokens.in" +#line 47 "xcsv_tokens.in" {"LAT_NMEA", XT_LAT_NMEA}, -#line 70 "xcsv_tokens.in" +#line 71 "xcsv_tokens.in" {"ROUTE_NAME", XT_ROUTE_NAME}, -#line 68 "xcsv_tokens.in" +#line 69 "xcsv_tokens.in" {"POSTAL_CODE", XT_POSTAL_CODE}, -#line 55 "xcsv_tokens.in" +#line 56 "xcsv_tokens.in" {"LON_NMEA", XT_LON_NMEA}, -#line 72 "xcsv_tokens.in" +#line 73 "xcsv_tokens.in" {"STATE", XT_STATE}, -#line 22 "xcsv_tokens.in" +#line 23 "xcsv_tokens.in" {"GEOCACHE_TERR", XT_GEOCACHE_TERR}, -#line 21 "xcsv_tokens.in" +#line 22 "xcsv_tokens.in" {"GEOCACHE_PLACER", XT_GEOCACHE_PLACER}, -#line 82 "xcsv_tokens.in" +#line 83 "xcsv_tokens.in" {"UTM", XT_UTM}, -#line 18 "xcsv_tokens.in" - {"GEOCACHE_ISARCHIVED", XT_GEOCACHE_ISARCHIVED}, #line 19 "xcsv_tokens.in" + {"GEOCACHE_ISARCHIVED", XT_GEOCACHE_ISARCHIVED}, +#line 20 "xcsv_tokens.in" {"GEOCACHE_ISAVAILABLE", XT_GEOCACHE_ISAVAILABLE}, #line 5 "xcsv_tokens.in" {"ANYNAME", XT_ANYNAME}, -#line 24 "xcsv_tokens.in" +#line 25 "xcsv_tokens.in" {"GMT_TIME", XT_GMT_TIME}, -#line 76 "xcsv_tokens.in" +#line 77 "xcsv_tokens.in" {"TIMET_TIME", XT_TIMET_TIME}, -#line 14 "xcsv_tokens.in" +#line 15 "xcsv_tokens.in" {"FORMAT", XT_FORMAT}, -#line 13 "xcsv_tokens.in" +#line 14 "xcsv_tokens.in" {"FILENAME", XT_FILENAME}, -#line 33 "xcsv_tokens.in" +#line 34 "xcsv_tokens.in" {"ICON_DESCR", XT_ICON_DESCR}, #line 10 "xcsv_tokens.in" {"DESCRIPTION", XT_DESCRIPTION}, -#line 16 "xcsv_tokens.in" +#line 17 "xcsv_tokens.in" {"GEOCACHE_DIFF", XT_GEOCACHE_DIFF}, -#line 15 "xcsv_tokens.in" +#line 16 "xcsv_tokens.in" {"GEOCACHE_CONTAINER", XT_GEOCACHE_CONTAINER}, -#line 79 "xcsv_tokens.in" +#line 80 "xcsv_tokens.in" {"TRACK_NEW", XT_TRACK_NEW}, -#line 78 "xcsv_tokens.in" +#line 79 "xcsv_tokens.in" {"TRACK_NAME", XT_TRACK_NAME}, -#line 81 "xcsv_tokens.in" +#line 82 "xcsv_tokens.in" {"URL", XT_URL}, -#line 66 "xcsv_tokens.in" +#line 67 "xcsv_tokens.in" {"PATH_SPEED", XT_PATH_SPEED}, -#line 59 "xcsv_tokens.in" +#line 60 "xcsv_tokens.in" {"PATH_COURSE", XT_PATH_COURSE}, -#line 17 "xcsv_tokens.in" +#line 18 "xcsv_tokens.in" {"GEOCACHE_HINT", XT_GEOCACHE_HINT}, -#line 39 "xcsv_tokens.in" +#line 40 "xcsv_tokens.in" {"LAT_DECIMAL", XT_LAT_DECIMAL}, #line 9 "xcsv_tokens.in" {"COUNTRY", XT_COUNTRY}, -#line 42 "xcsv_tokens.in" +#line 43 "xcsv_tokens.in" {"LAT_HUMAN_READABLE", XT_LAT_HUMAN_READABLE}, -#line 47 "xcsv_tokens.in" +#line 48 "xcsv_tokens.in" {"LOCAL_TIME", XT_LOCAL_TIME}, -#line 49 "xcsv_tokens.in" +#line 50 "xcsv_tokens.in" {"LON_DECIMAL", XT_LON_DECIMAL}, -#line 52 "xcsv_tokens.in" +#line 53 "xcsv_tokens.in" {"LON_HUMAN_READABLE", XT_LON_HUMAN_READABLE}, -#line 71 "xcsv_tokens.in" +#line 72 "xcsv_tokens.in" {"SHORTNAME", XT_SHORTNAME}, -#line 67 "xcsv_tokens.in" +#line 68 "xcsv_tokens.in" {"PHONE_NR", XT_PHONE_NR}, #line 7 "xcsv_tokens.in" {"CITY", XT_CITY}, -#line 74 "xcsv_tokens.in" +#line 75 "xcsv_tokens.in" {"TEMPERATURE", XT_TEMPERATURE}, -#line 37 "xcsv_tokens.in" +#line 38 "xcsv_tokens.in" {"ISO_TIME", XT_ISO_TIME}, #line 3 "xcsv_tokens.in" {"ALT_FEET", XT_ALT_FEET}, -#line 32 "xcsv_tokens.in" +#line 33 "xcsv_tokens.in" {"HMSL_TIME", XT_HMSL_TIME}, -#line 56 "xcsv_tokens.in" +#line 57 "xcsv_tokens.in" {"MAP_EN_BNG", XT_MAP_EN_BNG}, -#line 34 "xcsv_tokens.in" +#line 35 "xcsv_tokens.in" {"IGNORE", XT_IGNORE}, -#line 75 "xcsv_tokens.in" +#line 76 "xcsv_tokens.in" {"TEMPERATURE_F", XT_TEMPERATURE_F}, -#line 40 "xcsv_tokens.in" +#line 41 "xcsv_tokens.in" {"LAT_DIRDECIMAL", XT_LAT_DIRDECIMAL}, -#line 35 "xcsv_tokens.in" +#line 36 "xcsv_tokens.in" {"INDEX", XT_INDEX}, -#line 83 "xcsv_tokens.in" +#line 84 "xcsv_tokens.in" {"UTM_ZONE", XT_UTM_ZONE}, -#line 50 "xcsv_tokens.in" +#line 51 "xcsv_tokens.in" {"LON_DIRDECIMAL", XT_LON_DIRDECIMAL}, -#line 44 "xcsv_tokens.in" +#line 45 "xcsv_tokens.in" {"LATLON_HUMAN_READABLE", XT_LATLON_HUMAN_READABLE}, #line 8 "xcsv_tokens.in" {"CONSTANT", XT_CONSTANT}, -#line 20 "xcsv_tokens.in" +#line 21 "xcsv_tokens.in" {"GEOCACHE_LAST_FOUND", XT_GEOCACHE_LAST_FOUND}, -#line 11 "xcsv_tokens.in" +#line 12 "xcsv_tokens.in" {"EXCEL_TIME", XT_EXCEL_TIME}, -#line 86 "xcsv_tokens.in" +#line 87 "xcsv_tokens.in" {"UTM_EASTING", XT_UTM_EASTING}, -#line 23 "xcsv_tokens.in" +#line 24 "xcsv_tokens.in" {"GEOCACHE_TYPE", XT_GEOCACHE_TYPE}, -#line 85 "xcsv_tokens.in" +#line 86 "xcsv_tokens.in" {"UTM_ZONEF", XT_UTM_ZONEF}, -#line 88 "xcsv_tokens.in" +#line 89 "xcsv_tokens.in" {"YYYYMMDD_TIME", XT_YYYYMMDD_TIME}, -#line 31 "xcsv_tokens.in" +#line 32 "xcsv_tokens.in" {"HMSG_TIME", XT_HMSG_TIME}, -#line 60 "xcsv_tokens.in" +#line 61 "xcsv_tokens.in" {"PATH_DISTANCE_KM", XT_PATH_DISTANCE_KM}, -#line 12 "xcsv_tokens.in" +#line 13 "xcsv_tokens.in" {"FACILITY", XT_FACILITY}, -#line 80 "xcsv_tokens.in" +#line 81 "xcsv_tokens.in" {"URL_LINK_TEXT", XT_URL_LINK_TEXT}, -#line 84 "xcsv_tokens.in" +#line 85 "xcsv_tokens.in" {"UTM_ZONEC", XT_UTM_ZONEC}, -#line 25 "xcsv_tokens.in" +#line 26 "xcsv_tokens.in" {"GPS_FIX", XT_GPS_FIX}, -#line 77 "xcsv_tokens.in" +#line 78 "xcsv_tokens.in" {"TIMET_TIME_MS", XT_TIMET_TIME_MS}, -#line 57 "xcsv_tokens.in" +#line 58 "xcsv_tokens.in" {"NOTES", XT_NOTES}, -#line 65 "xcsv_tokens.in" +#line 66 "xcsv_tokens.in" {"PATH_SPEED_MPH", XT_PATH_SPEED_MPH}, -#line 64 "xcsv_tokens.in" +#line 65 "xcsv_tokens.in" {"PATH_SPEED_KPH", XT_PATH_SPEED_KPH}, -#line 36 "xcsv_tokens.in" +#line 37 "xcsv_tokens.in" {"ISO_TIME_MS", XT_ISO_TIME_MS}, -#line 29 "xcsv_tokens.in" +#line 30 "xcsv_tokens.in" {"GPS_VDOP", XT_GPS_VDOP}, #line 4 "xcsv_tokens.in" {"ALT_METERS", XT_ALT_METERS}, -#line 28 "xcsv_tokens.in" +#line 29 "xcsv_tokens.in" {"GPS_SAT", XT_GPS_SAT}, -#line 62 "xcsv_tokens.in" +#line 63 "xcsv_tokens.in" {"PATH_DISTANCE_MILES", XT_PATH_DISTANCE_MILES}, -#line 61 "xcsv_tokens.in" +#line 62 "xcsv_tokens.in" {"PATH_DISTANCE_METERS", XT_PATH_DISTANCE_METERS}, -#line 63 "xcsv_tokens.in" +#line 64 "xcsv_tokens.in" {"PATH_SPEED_KNOTS", XT_PATH_SPEED_KNOTS}, -#line 43 "xcsv_tokens.in" +#line 11 "xcsv_tokens.in" + {"EMAIL", XT_EMAIL}, +#line 44 "xcsv_tokens.in" {"LAT_INT32DEG", XT_LAT_INT32DEG}, -#line 53 "xcsv_tokens.in" +#line 54 "xcsv_tokens.in" {"LON_INT32DEG", XT_LON_INT32DEG}, -#line 87 "xcsv_tokens.in" +#line 88 "xcsv_tokens.in" {"UTM_NORTHING", XT_UTM_NORTHING}, -#line 26 "xcsv_tokens.in" - {"GPS_HDOP", XT_GPS_HDOP}, #line 27 "xcsv_tokens.in" + {"GPS_HDOP", XT_GPS_HDOP}, +#line 28 "xcsv_tokens.in" {"GPS_PDOP", XT_GPS_PDOP} }; @@ -300,11 +302,11 @@ Perfect_Hash::in_word_set (const char *str, size_t len) -1, -1, -1, -1, -1, -1, -1, -1, 70, -1, 71, -1, -1, -1, -1, -1, -1, -1, -1, 72, -1, -1, -1, -1, 73, -1, 74, -1, 75, -1, 76, -1, 77, -1, 78, 79, 80, -1, -1, -1, -1, -1, + -1, -1, -1, 81, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 81, -1, -1, -1, -1, 82, -1, - -1, -1, -1, 83, 84, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 85 + -1, -1, -1, -1, -1, -1, -1, 82, -1, -1, -1, -1, 83, -1, + -1, -1, -1, 84, 85, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 86 }; if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) diff --git a/xcsv_tokens.in b/xcsv_tokens.in index 5f3637d88..aced51e37 100644 --- a/xcsv_tokens.in +++ b/xcsv_tokens.in @@ -8,6 +8,7 @@ CITY, XT_CITY CONSTANT, XT_CONSTANT COUNTRY, XT_COUNTRY DESCRIPTION, XT_DESCRIPTION +EMAIL, XT_EMAIL EXCEL_TIME, XT_EXCEL_TIME FACILITY, XT_FACILITY FILENAME, XT_FILENAME diff --git a/xmldoc/chapters/styles.xml b/xmldoc/chapters/styles.xml index 5a4f8b43a..1fd535470 100644 --- a/xmldoc/chapters/styles.xml +++ b/xmldoc/chapters/styles.xml @@ -1208,6 +1208,18 @@ longitude) COUNTRY, "", "%s" +
+ EMAIL + An email address associated with a position. + example: + + + + + + + EMAIL, "", "%s" +
FACILITY The name of a facility to associate with a position. @@ -1352,4 +1364,4 @@ look like:
- \ No newline at end of file + -- 2.30.2